
# set working directory
setwd(dirname(rstudioapi::getActiveDocumentContext()$path))

library("tidyverse") # Easily Install and Load the 'Tidyverse', CRAN v1.3.1
library("quanteda") # Quantitative Analysis of Textual Data, CRAN v3.2.1
library("lubridate") # Make Dealing with Dates a Little Easier, CRAN v1.8.0
library("stringr") # Simple, Consistent Wrappers for Common String Operations, CRAN v1.4.0
library("knitr") # A General-Purpose Package for Dynamic Report Generation in R, CRAN v1.37
library("kableExtra") # Construct Complex Table with 'kable' and Pipe Syntax, CRAN v1.3.4
#library("quanteda.dictionaries") # Dictionaries for Text Analysis and Associated Utilities, [github::kbenoit/quanteda.dictionaries] v0.31
library("cowplot") # Streamlined Plot Theme and Plot Annotations for 'ggplot2', CRAN v1.1.1 
library("extrafont") # Tools for using fonts, CRAN v0.17
library("haven") # Import and Export 'SPSS', 'Stata' and 'SAS' Files, CRAN v2.4.3
library("texreg") # Conversion of R Regression Output to LaTeX or HTML Tables, CRAN v1.37.5
library("DT") # A Wrapper of the JavaScript Library 'DataTables', CRAN v0.20
library("viridis") # Colorblind-Friendly Color Maps for R, CRAN v0.6.2
library("quanteda.textstats") # Textual Statistics for the Quantitative Analysis of Textual Data, CRAN v0.95
library("readxl") # Read Excel Files, CRAN v1.3.1
library("plotrix") # Various Plotting Functions, CRAN v3.8-2


source("dictionary_postcoding.R", encoding = "UTF-8")

dimensions <- c("competition","participation","representation","transparency","rule_of_law","constraints","public_sphere","ind_liberties")
names(dimensions)<-c("Competition","Participation","Representation","Transparency", "Rule of Law","Constraints","Public Sphere","Ind. Liberties")




party_names_at<-c("FPO","Greens","NEOS","OVP","Pilz","SPO")
party_names_de<-c("AfD","CDU","CSU","FDP","Greens","Left Party","SPD")
party_names_ch<-c("BDP","CVP","FDP","Greenliberals","Greens","SP","SVP")
names(party_names_at)<-c("fpo","gruene","neos","ovp","pilz","spo")
names(party_names_de)<-c("afd","cdu","csu","fdp","gruene","linke","spd")
names(party_names_ch)<-c("bdp","cvp","fdp_ch","greenliberals","greens_ch","sp","svp")

colors <- viridis(n=3, alpha = 1, begin = 0, end = 0.7, direction = -1, option = "D")
colors <- c(colors[2],colors[1],colors[3])

# download press releases at https://dataverse.harvard.edu/dataset.xhtml?persistentId=doi:10.7910/DVN/TXKO19
load("pressreleases_DACH.RData")
all$year<- as.numeric(format(all$date, "%Y"))
all<-subset(all,str_detect(all$title,"Interview")==FALSE)
all<-subset(all,duplicated(all$text)==F)
all$ym <- format(all$date,"%m/%y")

load("MP_de.RData")
load("ministers_de.RData")

load("MP_ch.RData")
load("ministers_ch.RData")

load("MP_at.RData")
load("ministers_at.RData")

at <- subset(all,country=="AT")
de <- subset(all,country=="DE")
ch <- subset(all,country=="CH")
de<-subset(de,duplicated(de$text)==F)
ch<-subset(ch,duplicated(ch$text)==F)
at<-subset(at,duplicated(at$text)==F)

at <- at %>%
  mutate(party = ifelse(party == "gruene_at", "gruene", party))

de <- de %>% 
  mutate(party = ifelse(party == "fdp_de", "fdp", party)) %>% 
  mutate(party = ifelse(party == "gruene_de", "gruene", party))

ch <- ch %>% 
  mutate(party = ifelse(party == "gruene_ch", "greens_ch", party)) %>% 
  mutate(party = ifelse(party == "gruenliberale", "greenliberals", party))

at_overview <- at %>% group_by(party,ym) %>%
  summarise(releases_n=n(),first=min(date),last=max(date)) %>% 
  group_by(party)%>%
  summarise(releases=sum(releases_n),avg=mean(releases_n),first=min(first),last=max(last))

de_overview <- de %>% group_by(party,ym) %>%
  summarise(releases_n=n(),first=min(date),last=max(date)) %>% 
  group_by(party)%>%
  summarise(releases=sum(releases_n),avg=mean(releases_n),first=min(first),last=max(last))

ch_overview <- ch %>% group_by(party,ym) %>%
  summarise(releases_n=n(),first=min(date),last=max(date)) %>% 
  group_by(party)%>%
  summarise(releases=sum(releases_n),avg=mean(releases_n),first=min(first),last=max(last))

summary_stats<-bind_rows(list(Austria=at_overview,Germany=de_overview,Switzerland=ch_overview),.id="id") 

summary_stats %>% mutate(range=paste(format(first,"%m/%y"),"-",format(last,"%m/%y")))  %>% 
  select(party,range,releases) %>% mutate(party=str_to_upper(party)) %>%
  kable(booktab = T, label="nopr", caption = "Number and time period of press releases", digits=1, col.names = c("Party","period", "N"))%>% kableExtra::kable_styling() %>%
  group_rows(index=table(summary_stats$id))

kable <- summary_stats %>% mutate(range=paste(format(first,"%m/%y"),"-",format(last,"%m/%y")))  %>% 
  select(party,range,releases) %>% mutate(party=str_to_upper(party)) %>%
  kable(booktab = T,format="latex", label="nopr", caption = "Number and time period of press releases", digits=1, col.names = c("Party","period", "N"))%>% kableExtra::kable_styling() %>%
  group_rows(index=table(summary_stats$id))

dir.create("temp")
write.csv(summary_stats, "temp/Table2.csv")
writeLines(kable, 'temp/Table2.tex')
# ## Sample export
# sampling<- at %>% filter(party!="pilz") %>% bind_rows(ch,de)
# set.seed(2)
# drawn_sample <- sampling %>%
#   group_by(party,country) %>%
#   sample_n(size=40)
# 
# sample_toks<-drawn_sample %>%
#   corpus() %>%
#   tokens(
#     remove_punct = T,
#     remove_symbols = T,
#     remove_separators = T,
#     split_hyphens = T,
#     remove_numbers = T,
#     remove_url = T) %>%
#   tokens_remove("\\d") %>% tokens_remove(c(stopwords("de"),"dass")) %>%
#   tokens_tolower() %>%
#   tokens_split(".") %>% 
#   tokens_remove(ministers_at) %>% 
#   tokens_remove(MP_at)
# 
# sample_toks_eval<-sample_toks %>% tokens_lookup(dict_prinzip,level=2)
# 
# 
# sample_dist<-sample_toks_eval %>% dfm() %>% dfm_weight("boolean") %>% convert("data.frame")
# 
# #save(sample_dist,sample_toks_eval,drawn_sample,file="../Data/sample.RData")
# 
# export <- drawn_sample %>% select(title,text,date,country,party) %>%
#   mutate(public_sphere="",
#          ind_liberties="",
#          rule_of_law="",
#          constraints="",
#          transparency="",
#          representation="",
#          participation="",
#          competition="")
# 
# 
# 
# #Häufigkeit der Begriffe im sample
# 
# load("sample.RData")
# sample_dist %>% select(-document) %>% summarize_all(sum) %>%
#   kable() %>% kable_styling()


## Preprocessing


dimensions <- c("competition","participation","representation","transparency","rule_of_law","constraints","public_sphere","ind_liberties")
names(dimensions)<-c("Competition","Participation","Representation","Transparency", "Rule of Law","Constraints","Public Sphere","Ind. Liberties")
party_names_at<-c("FPO","Greens","NEOS","OVP","Pilz","SPO")
party_names_de<-c("AfD","CDU","CSU","FDP","Greens","Left Party","SPD")
party_names_ch<-c("BDP","CVP","FDP","Greenliberals","Greens","SP","SVP")
names(party_names_at)<-c("fpo","gruene","neos","ovp","pilz","spo")
names(party_names_de)<-c("afd","cdu","csu","fdp","gruene","linke","spd")
names(party_names_ch)<-c("bdp","cvp","fdp_ch","greenliberals","greens_ch","sp","svp")

at_corp <- corpus(at, text_field = "text")
at_toks <- tokens(at_corp,
                  remove_punct = T,
                  remove_symbols = T,
                  remove_separators = T,
                  split_hyphens = T,
                  remove_numbers = T,
                  remove_url = T) %>%
  tokens_remove("\\d") %>% tokens_remove(c(stopwords("de"),"dass")) %>%
  tokens_tolower() %>%
  tokens_split(".") %>% 
  tokens_remove(ministers_at) %>% 
  tokens_remove(MP_at)

at_toks_prinzip<-at_toks %>% tokens_lookup(dict_prinzip,level=2)

at_count_prinzip_detail<-at_toks_prinzip %>% dfm() %>% dfm_weight("boolean") %>% convert("data.frame")

at_salience_df <- at_count_prinzip_detail %>% bind_cols(docvars(at_toks_prinzip)) %>%
  select(party,public_sphere:competition) %>% group_by(party) %>%
  summarize_all(mean)

at_agg_salience_df <- at_count_prinzip_detail %>% bind_cols(docvars(at_toks_prinzip)) %>%
  select(public_sphere:competition) %>% 
  summarize_all(mean) %>%
  gather("principle","salience")

count_df <- at_count_prinzip_detail %>% bind_cols(docvars(at_toks_prinzip)) %>%
  group_by(party) 

count_df_hy<- count_df %>% mutate(quarter=lubridate::round_date(date,unit="halfyear"),n=1) %>%
  group_by(party,quarter)  %>% 
  select(public_sphere:competition,n) %>% summarize_all(sum) %>%
  mutate_at(vars(public_sphere:competition),funs(./n))


count_df_q<- count_df %>% mutate(quarter=lubridate::round_date(date,unit="quarter"),n=1) %>%
  group_by(party,quarter)  %>% 
  select(public_sphere:competition,n) %>% summarize_all(sum) %>%
  mutate_at(vars(public_sphere:competition),funs(./n))

save(count_df_q,count_df_hy,file="temp/at_agg1.RData")


de_corp <- corpus(de, text_field = "text")
de_toks <- tokens(de_corp,
                  remove_punct = T,
                  remove_symbols = T,
                  remove_separators = T,
                  split_hyphens = T,
                  remove_numbers = T,
                  remove_url = T) %>%
  tokens_remove("\\d") %>% tokens_remove(c(stopwords("de"),"dass")) %>%
  tokens_tolower() %>%
  tokens_split(".") %>% 
  tokens_remove(ministers_de) %>% 
  tokens_remove(MP_de)

de_toks_prinzip<-de_toks %>% tokens_lookup(dict_prinzip,level=2)

de_count_prinzip_detail<-de_toks_prinzip %>% dfm() %>% dfm_weight("boolean") %>% convert("data.frame")

de_salience_df <- de_count_prinzip_detail %>% bind_cols(docvars(de_toks_prinzip)) %>%
  select(party,public_sphere:competition) %>% group_by(party) %>%
  summarize_all(mean)

de_agg_salience_df <- de_count_prinzip_detail %>% bind_cols(docvars(de_toks_prinzip)) %>%
  select(public_sphere:competition) %>% 
  summarize_all(mean) %>%
  gather("principle","salience")

count_df <- de_count_prinzip_detail %>% bind_cols(docvars(de_toks_prinzip)) %>%
  group_by(party) 

count_df_hy<- count_df %>% mutate(quarter=lubridate::round_date(date,unit="halfyear"),n=1) %>%
  group_by(party,quarter)  %>% 
  select(public_sphere:competition,n) %>% summarize_all(sum) %>%
  mutate_at(vars(public_sphere:competition),funs(./n))


count_df_q<- count_df %>% mutate(quarter=lubridate::round_date(date,unit="quarter"),n=1) %>%
  group_by(party,quarter)  %>% 
  select(public_sphere:competition,n) %>% summarize_all(sum) %>%
  mutate_at(vars(public_sphere:competition),funs(./n))

save(count_df_q,count_df_hy,file="temp/de_agg1.RData")

ch_corp <- corpus(ch, text_field = "text")
ch_toks <- tokens(ch_corp,
                  remove_punct = T,
                  remove_symbols = T,
                  remove_separators = T,
                  split_hyphens = T,
                  remove_numbers = T,
                  remove_url = T) %>%
  tokens_remove("\\d") %>% tokens_remove(c(stopwords("de"),"dass")) %>%
  tokens_tolower() %>%
  tokens_split(".") %>% 
  tokens_remove(ministers_ch) %>% 
  tokens_remove(MP_ch)

ch_toks_prinzip<-ch_toks %>% tokens_lookup(dict_prinzip,level=2)

ch_count_prinzip_detail<-ch_toks_prinzip %>% dfm() %>% dfm_weight("boolean") %>% convert("data.frame")

ch_salience_df <- ch_count_prinzip_detail %>% bind_cols(docvars(ch_toks_prinzip)) %>%
  select(party,public_sphere:competition) %>% group_by(party) %>%
  summarize_all(mean)

ch_agg_salience_df <- ch_count_prinzip_detail %>% bind_cols(docvars(ch_toks_prinzip)) %>%
  select(public_sphere:competition) %>% 
  summarize_all(mean) %>%
  gather("principle","salience")

count_df <- ch_count_prinzip_detail %>% bind_cols(docvars(ch_toks_prinzip)) %>%
  group_by(party) 

count_df_hy<- count_df %>% mutate(quarter=lubridate::round_date(date,unit="halfyear"),n=1) %>%
  group_by(party,quarter)  %>% 
  select(public_sphere:competition,n) %>% summarize_all(sum) %>%
  mutate_at(vars(public_sphere:competition),funs(./n))


count_df_q<- count_df %>% mutate(quarter=lubridate::round_date(date,unit="quarter"),n=1) %>%
  group_by(party,quarter)  %>% 
  select(public_sphere:competition,n) %>% summarize_all(sum) %>%
  mutate_at(vars(public_sphere:competition),funs(./n))

save(count_df_q,count_df_hy,file="temp/ch_agg1.RData")



## Salience Plots

create_df <- function(x){
  x %>% 
    gather("princ","salience",-party) %>% 
    left_join(data.frame(princ=dimensions,principle=names(dimensions))) %>%
    mutate(dimension=ifelse(principle %in% c("Competition","Participation","Representation","Transparency"),"Participatory Principles (incl. Transparency)","Liberal Principles")) %>%
    mutate(principle=reorder(principle,desc(principle))) %>%
    mutate(alpha_var=ifelse(party %in% c("fpo","gruene","afd","svp","greens_ch"),1,0))
}


plot_salience <- function(x){
  x %>%
    ggplot(aes(x=salience,y=principle,
               color=party_dummy,#alpha=alpha_var,
               shape=party_dummy,size=alpha_var)) + 
    geom_point(position=position_nudge(y=x$alpha_var/5-0.1)) + scale_shape_manual(values=c(15,17,16))+
    #scale_alpha_continuous(range=c(0.4,1))+
    scale_size(range=c(1,3))+
    scale_color_manual(name="Party", values=colors)+
    theme_minimal()+theme(legend.position="bottom")+
    guides(alpha="none",shape="none",size="none",color=guide_legend(override.aes = list(shape=c(15,17,16),size=5)))+
    theme_minimal()+facet_wrap(~dimension,ncol=1,scales="free_y")+
    xlab("Salience")+ylab("")+theme(panel.grid.major.y = element_blank(),
                                    panel.grid.minor.y = element_blank(),
                                    legend.position="bottom",text=element_text(size=20))
}


at_plot <- create_df(at_salience_df)  %>%
  mutate(party_dummy=factor(case_when(party=="fpo"~"FPÖ",
                                      party=="gruene"~"Greens",
                                      TRUE~"other parties"),levels=c("FPÖ","Greens","other parties")))  %>% plot_salience()

ann_text <- data.frame(salience=0.15, principle=1, label="NEOS",
                       dimension="Participatory Principles (incl. Transparency)",party_dummy="other parties",alpha_var=1, xendf = 0.173
) %>%
  bind_rows(data.frame(salience=0.12, principle=1, label="Liste PILZ",
                       dimension="Participatory Principles (incl. Transparency)",party_dummy="other parties",alpha_var=1,xendf=0.095))


at_plot <- at_plot+
  geom_curve(data=ann_text,
             aes(x =salience , y = principle, xend = xendf, yend = "Transparency"),
             curvature = .2, arrow = arrow(length = unit(2, "mm")),size=0.5, show_guide = FALSE)+
  geom_label(data =ann_text,
             aes(x=salience,y=principle,label=label,label.size=0),show_guide = F,size=3)


de_plot <- create_df(de_salience_df)  %>%
  mutate(party_dummy=factor(case_when(party=="afd"~"AfD",
                                      party=="gruene"~"Greens",
                                      TRUE~"other parties"), levels=c("AfD","Greens","other parties"))) %>% plot_salience()

#ann_text <- data.frame(salience=0.05, principle=2.4, label="FDP",
#dimension="Liberal Principles")
#de_plot + geom_text(data =ann_text,
#aes(x=salience,y=principle,label=label),
#color="blue",inherit.aes=F)
# fdp rule of law public sphere 0.141 0.832


ann_text <- data.frame(salience=0.1, principle=2, label="FDP",
                       dimension="Liberal Principles",party_dummy="other parties",alpha_var=1)
de_plot <- de_plot+
  geom_curve(data=ann_text,
             aes(x =salience , y = principle, xend = 0.139, yend = "Rule of Law"),show_guide=F,
             curvature = .2, arrow = arrow(length = unit(2, "mm")),size=0.5)+
  geom_curve(data=ann_text,
             aes(x =salience , y = principle, xend = 0.084, yend = "Public Sphere"),show_guide=F,
             curvature = .2, arrow = arrow(length = unit(2, "mm")),size=0.5) + 
  geom_label(data =ann_text,
             aes(x=salience,y=principle,label=label,label.size=0),show.legend = F,size=5,show_guide=F)

ch_plot <- create_df(ch_salience_df)  %>%
  mutate(party_dummy=factor(case_when(party=="svp"~"SVP",
                                      party=="greens_ch"~"Greens",
                                      TRUE~"other parties"),levels=c("SVP","Greens","other parties"))) %>% plot_salience()


ann_text <- data.frame(salience=0.15, principle=1, label="SP",
                       dimension="Participatory Principles (incl. Transparency)",party_dummy="other parties",alpha_var=1)

ch_plot <- ch_plot+
  geom_curve(data=ann_text,
             aes(x =salience , y = principle, xend = 0.11, yend = "Transparency"),
             curvature = .2, arrow = arrow(length = unit(2, "mm")),size=0.5, show_guide = FALSE)+
  geom_label(data =ann_text,
             aes(x=salience,y=principle,label=label,label.size=0),show_guide = F,size=5)



ggsave(filename="Figure2.pdf",
       path="temp",
       plot=at_plot,
       dpi=300,
       units="cm",
       width=17.78,
       height=12.66)
ggsave(filename="Figure3.pdf",
       path="temp",
       plot=de_plot,
       dpi=300,
       units="cm",
       width=17.78,
       height=12.66)
ggsave(filename="Figure1.pdf",
       path="temp",
       plot=ch_plot,
       dpi=300,
       units="cm",
       width=17.78,
       height=12.66)




## Salience Plots for Appendix


ch_salience_df_un <- ch_count_prinzip_detail %>% bind_cols(docvars(ch_toks_prinzip)) %>%
  select(party,public_sphere:competition) %>% group_by(party) %>%
  summarize_all(list(mean,std.error))

de_salience_df_un  <- de_count_prinzip_detail %>% bind_cols(docvars(de_toks_prinzip)) %>%
  select(party,public_sphere:competition) %>% group_by(party) %>%
  summarize_all(list(mean,std.error))

at_salience_df_un  <- at_count_prinzip_detail %>% bind_cols(docvars(at_toks_prinzip)) %>%
  select(party,public_sphere:competition) %>% group_by(party) %>%
  summarize_all(list(mean,std.error))



create_df_2 <- function(x){
  x  %>% 
    gather("princ","salience",-party )%>% 
    separate(princ,into=c("princ","fn"),sep="_fn") %>%
    left_join(data.frame(princ=dimensions,principle=names(dimensions))) %>% mutate(fn=case_when(fn==1~"mean",fn==2~"se")) %>%
    spread(fn,salience) %>%
    mutate(dimension=ifelse(principle %in% c("Competition","Participation","Representation","Transparency"),"Participatory Principles (incl. Transparency)","Liberal Principles")) %>%
    mutate(principle=reorder(principle,desc(principle))) %>%
    mutate(alpha_var=ifelse(party %in% c("fpo","gruene","afd","svp","greens_ch"),1,0))
}

de_plot2 <- create_df_2(de_salience_df_un)
at_plot2 <- create_df_2(at_salience_df_un)
ch_plot2 <- create_df_2(ch_salience_df_un)



plot_salience_2 <- function(x){
  x %>%
    ggplot(aes(x=mean,y=principle,
               color=party,#alpha=alpha_var,
               shape=factor(alpha_var),group=party)) + 
    geom_linerange(aes(xmin=mean-1.96*se,xmax=mean+1.96*se),position=position_dodge(0.5))+
    geom_point(position=position_dodge(0.5)) + 
    theme_minimal()+facet_wrap(~dimension,ncol=1,scales="free_y")+
    xlab("Salience")+ylab("")+theme(panel.grid.major.y = element_blank(),
                                    panel.grid.minor.y = element_blank(),
                                    legend.position="bottom",text=element_text(size=15))+
    scale_color_brewer(name="Party",type="qual")+
    guides(shape=F)
}

de_plot_app <- de_plot2 %>%
  mutate(party=recode(party,!!!party_names_de)) %>%
  plot_salience_2()
at_plot_app <- at_plot2 %>%
  mutate(party=recode(party,!!!party_names_at)) %>%
  plot_salience_2()
ch_plot_app <- ch_plot2 %>%
  mutate(party=recode(party,!!!party_names_ch)) %>%
  plot_salience_2()


ggsave(filename="App_Figure3.pdf",
       path="temp",
       plot=de_plot_app,
       dpi=300,
       units="cm",
       width=17.78,
       height=12.66)


ggsave(filename="App_Figure2.pdf",
       path="temp",
       plot = at_plot_app,
       dpi=300,
       units="cm",
       width=17.78,
       height=12.66)


ggsave(filename="App_Figure1.pdf",
       path="temp",
       plot = ch_plot_app,
       dpi=300,
       units="cm",
       width=17.78,
       height=12.66)



## Contrast



see_contrast <- function(object=toks,prinzip=prinzip,populistparty="afd"){
  kwic<- kwic(x=object,phrase(prinzip), window = 10) %>% data.frame() %>%
    left_join(docvars(object),by="docname") %>% 
    mutate(text=paste(pre,pattern,post),docid=docname,pattern) %>%
    corpus %>% dfm() %>% dfm_group(groups=party) %>%
    dfm_trim(5) %>% dfm_remove("*",valuetype="fixed") %>%
    textstat_keyness(target=populistparty) 
}


df<-de %>% subset(party=="gruene" | party=="afd")
# str_replace_all(df$text, "ß","ss") (R crashes down - currently: words with ss and ß in dictionary)

extra_stopwords<-c("afd","cdu","csu","spd","linke","drucken","pressemitteilungen","fdp","grüne","prozent",
                   "schlagworte","startseite","buvo","bundestagsfraktion","fraktion","partei","die","sprecherin",
                   "alternative","deutschland","union","bundestag","bundesregierung",
                   "cdata","ots","meuthen","lucke","poggenburg","henkel","pazderski","adam","stellvertretende","bundesvorstandsmitglied","vorsitzende","vorsitzender","dtal","jquery","sprecher",
                   "sei","sagte","forder*","betonte","betont","müsse","dr","presseraum","pressekonferenz","aviso","uhr","seien","braucht","brauchen","*sprecher*","wurde","könne","sagen","hätten","sagt","document","hätten")
corp <- corpus(df, text_field = "text")
toks <- tokens(corp,
               remove_punct = T,
               remove_symbols = T,
               remove_separators = T,
               split_hyphens = T,
               remove_numbers = T,
               remove_url = T) %>%
  tokens_tolower() %>%
  tokens_split(".") %>% 
  tokens_remove("\\d") %>% tokens_remove(c(stopwords("de"),"dass")) %>%
  tokens_remove(extra_stopwords) %>%
  tokens_remove(ministers_de) %>% 
  tokens_remove(MP_de)




docvars(toks,"docname")<-docnames(toks)



public_sphere<-see_contrast(object=toks,
                            prinzip=dict_prinzip$prinzip$public_sphere,
                            populistparty="afd")
rule_of_law<-see_contrast(object=toks,
                          prinzip=dict_prinzip$prinzip$rule_of_law,
                          populistparty="afd")
competition<-see_contrast(object=toks,
                          prinzip=dict_prinzip$prinzip$competition,
                          populistparty="afd")
constraints<-see_contrast(object=toks,
                          prinzip=dict_prinzip$prinzip$constraints,
                          populistparty="afd")
ind_liberties<-see_contrast(object=toks,
                            prinzip=dict_prinzip$prinzip$ind_liberties,
                            populistparty="afd")
representation<-see_contrast(object=toks,
                             prinzip=dict_prinzip$prinzip$representation,
                             populistparty="afd")
participation<-see_contrast(object=toks,
                            prinzip=dict_prinzip$prinzip$participation,
                            populistparty="afd")
demokratie<-see_contrast(object=toks,
                         prinzip=dict_prinzip$prinzip$demokratie,
                         populistparty="afd")
transparency<-see_contrast(object=toks,
                           prinzip=dict_prinzip$prinzip$transparency,
                           populistparty="afd")
save(public_sphere,rule_of_law,competition,constraints,ind_liberties,
     representation,participation,demokratie,transparency,file="temp/afd_greens_context.RData")



df<-at %>% subset(party=="gruene" | party=="fpo")

extra_stopwords<-c("fpö","hc","nabg","grüne","övp","neos","spö","pk","karas","grünen",
                   "bundesparteiobmann","klubobmann","klubobfrau","ots","freiheitliche*","generalsekretär","sprecher*",
                   "sei","sagte","forder*","betonte","betont","müsse","dr","presseraum","pressekonferenz","aviso","uhr","seien","braucht","brauchen","*sprecher*","wurde","könne","sagen","hätten","sagt","document","hätten")

corp <- corpus(df, text_field = "text")
toks <- tokens(corp,
               remove_punct = T,
               remove_symbols = T,
               remove_separators = T,
               split_hyphens = T,
               remove_numbers = T,
               remove_url = T) %>%
  tokens_tolower() %>%
  tokens_split(".") %>% 
  tokens_remove("\\d") %>% tokens_remove(c(stopwords("de"),"dass")) %>%
  tokens_remove(extra_stopwords) %>%
  tokens_remove(ministers_at) %>% 
  tokens_remove(MP_at)

docvars(toks,"docname")<-docnames(toks)



public_sphere<-see_contrast(object=toks,
                            prinzip=dict_prinzip$prinzip$public_sphere,
                            populistparty="fpo")
rule_of_law<-see_contrast(object=toks,
                          prinzip=dict_prinzip$prinzip$rule_of_law,
                          populistparty="fpo")
competition<-see_contrast(object=toks,
                          prinzip=dict_prinzip$prinzip$competition,
                          populistparty="fpo")
constraints<-see_contrast(object=toks,
                          prinzip=dict_prinzip$prinzip$constraints,
                          populistparty="fpo")
ind_liberties<-see_contrast(object=toks,
                            prinzip=dict_prinzip$prinzip$ind_liberties,
                            populistparty="fpo")
representation<-see_contrast(object=toks,
                             prinzip=dict_prinzip$prinzip$representation,
                             populistparty="fpo")
participation<-see_contrast(object=toks,
                            prinzip=dict_prinzip$prinzip$participation,
                            populistparty="fpo")
demokratie<-see_contrast(object=toks,
                         prinzip=dict_prinzip$prinzip$demokratie,
                         populistparty="fpo")
transparency<-see_contrast(object=toks,
                           prinzip=dict_prinzip$prinzip$transparency,
                           populistparty="fpo")
save(public_sphere,rule_of_law,competition,constraints,ind_liberties,
     representation,participation,demokratie,transparency,file="temp/fpo_greens_context.RData")



df<-ch %>% subset(party=="greens_ch" | party=="svp")
# str_replace_all(df$text, "ß","ss") (R crashes down - currently: words with ss and ß in dictionary)


extra_stopwords<-c("svp","cvp","grünen","sp","bundesrat","bundesrätin","medienmitteilung","fdp","nationalrat","liberalen",
                   "bdp","mail@bdp","info","fax","tel","nr","partei","nationalrätin","grüne",
                   "sei","sagte","forder*","betonte","betont","müsse","dr","presseraum","pressekonferenz","aviso","uhr","seien","braucht","brauchen","*sprecher*","wurde","könne","sagen","hätten","sagt","document","hätten")
corp <- corpus(df, text_field = "text")
toks <- tokens(corp,
               remove_punct = T,
               remove_symbols = T,
               remove_separators = T,
               split_hyphens = T,
               remove_numbers = T,
               remove_url = T) %>%
  tokens_tolower() %>%
  tokens_split(".") %>% 
  tokens_remove("\\d") %>% tokens_remove(c(stopwords("de"),"dass")) %>%
  tokens_remove(extra_stopwords) %>%
  tokens_remove(ministers_ch) %>% 
  tokens_remove(MP_ch)



docvars(toks,"docname")<-docnames(toks)



public_sphere<-see_contrast(object=toks,
                            prinzip=dict_prinzip$prinzip$public_sphere,
                            populistparty="svp")
rule_of_law<-see_contrast(object=toks,
                          prinzip=dict_prinzip$prinzip$rule_of_law,
                          populistparty="svp")
competition<-see_contrast(object=toks,
                          prinzip=dict_prinzip$prinzip$competition,
                          populistparty="svp")
constraints<-see_contrast(object=toks,
                          prinzip=dict_prinzip$prinzip$constraints,
                          populistparty="svp")
ind_liberties<-see_contrast(object=toks,
                            prinzip=dict_prinzip$prinzip$ind_liberties,
                            populistparty="svp")
representation<-see_contrast(object=toks,
                             prinzip=dict_prinzip$prinzip$representation,
                             populistparty="svp")
participation<-see_contrast(object=toks,
                            prinzip=dict_prinzip$prinzip$participation,
                            populistparty="svp")
demokratie<-see_contrast(object=toks,
                         prinzip=dict_prinzip$prinzip$demokratie,
                         populistparty="svp")
transparency<-see_contrast(object=toks,
                           prinzip=dict_prinzip$prinzip$transparency,
                           populistparty="svp")
save(public_sphere,rule_of_law,competition,constraints,ind_liberties,
     representation,participation,demokratie,transparency,file="temp/svp_greens_context.RData")


load("temp/fpo_greens_context.RData")

at_competition <- competition
at_constraints <- constraints
at_demokratie <- demokratie
at_ind_liberties <- ind_liberties
at_participation <- participation
at_public_sphere <- public_sphere
at_representation <- representation
at_rule_of_law <- rule_of_law
at_transparency <- transparency

load("temp/svp_greens_context.RData")

ch_competition <- competition
ch_constraints <- constraints
ch_demokratie <- demokratie
ch_ind_liberties <- ind_liberties
ch_participation <- participation
ch_public_sphere <- public_sphere
ch_representation <- representation
ch_rule_of_law <- rule_of_law
ch_transparency <- transparency

load("temp/afd_greens_context.RData")

de_competition <- competition
de_constraints <- constraints
de_demokratie <- demokratie
de_ind_liberties <- ind_liberties
de_participation <- participation
de_public_sphere <- public_sphere
de_representation <- representation
de_rule_of_law <- rule_of_law
de_transparency <- transparency




n_at_transparency <- at_transparency %>%  head(10) %>% bind_rows(at_transparency %>% tail(10),.id="party")  
n_at_transparency$party <-  factor(recode(n_at_transparency$party, "1" = "FPÖ", "2" = "Grüne"))
n_at_participation <- at_participation %>% head(10) %>% bind_rows(at_participation %>% tail(10),.id="party") 
n_at_participation$party <-  recode(n_at_participation$party, "1" = "FPÖ", "2" = "Grüne")
n_at_ind_liberties <- at_ind_liberties %>% head(10) %>% bind_rows(at_ind_liberties %>% tail(10),.id="party") 
n_at_ind_liberties$party <-  recode(n_at_ind_liberties$party, "1" = "FPÖ", "2" = "Grüne")
n_at_public_sphere <- at_public_sphere %>% head(10) %>% bind_rows(at_public_sphere %>% tail(10),.id="party") 
n_at_public_sphere$party <-  recode(n_at_public_sphere$party, "1" = "FPÖ", "2" = "Grüne")



n_de_transparency <- de_transparency %>% head(10) %>% bind_rows(de_transparency %>% tail(10),.id="party") 
n_de_transparency$party <- recode(n_de_transparency$party, "1" = "AfD", "2" = "Grüne")
n_de_participation <- de_participation %>% head(10) %>% bind_rows(de_participation %>% tail(10),.id="party") 
n_de_participation$party <- recode(n_de_participation$party, "1" = "AfD", "2" = "Grüne")
n_de_ind_liberties <- de_ind_liberties %>% head(10) %>% bind_rows(de_ind_liberties %>% tail(10),.id="party") 
n_de_ind_liberties$party <- recode(n_de_ind_liberties$party, "1" = "AfD", "2" = "Grüne")
n_de_public_sphere <- de_public_sphere %>% head(10) %>% bind_rows(de_public_sphere %>% tail(10),.id="party")
n_de_public_sphere$party <- recode(n_de_public_sphere$party, "1" = "AfD", "2" = "Grüne")


n_ch_transparency <- ch_transparency %>% head(10) %>% bind_rows(ch_transparency %>% tail(10),.id="party") 
n_ch_transparency$party <- recode(n_ch_transparency$party, "1" = "SVP", "2" = "Grüne")
n_ch_participation <- ch_participation %>% head(10) %>% bind_rows(ch_participation %>% tail(10),.id="party") 
n_ch_participation$party <- recode(n_ch_participation$party, "1" = "SVP", "2" = "Grüne")
n_ch_ind_liberties <- ch_ind_liberties %>% head(10) %>% bind_rows(ch_ind_liberties %>% tail(10),.id="party") 
n_ch_ind_liberties$party <- recode(n_ch_ind_liberties$party, "1" = "SVP", "2" = "Grüne")
n_ch_public_sphere <- ch_public_sphere %>% head(10) %>% bind_rows(ch_public_sphere %>% tail(10),.id="party") 
n_ch_public_sphere$party <- recode(n_ch_public_sphere$party, "1" = "SVP", "2" = "Grüne")

n_trans <- bind_rows(list(Austria=n_at_transparency,Germany=n_de_transparency,Switzerland=n_ch_transparency),.id="country")


n_partip <- bind_rows(list(Austria=n_at_participation,Germany=n_de_participation,Switzerland=n_ch_participation),.id="country")


n_ind <- bind_rows(list(Austria=n_at_ind_liberties,Germany=n_de_ind_liberties,Switzerland=n_ch_ind_liberties),.id="country")


n_public <- bind_rows(list(Austria=n_at_public_sphere,Germany=n_de_public_sphere,Switzerland=n_ch_public_sphere),.id="country")


n_ch_public_sphere$party <- as.ordered(n_ch_public_sphere$party)
plot_contrast<-function(dat=n_ch_public_sphere,val1="SVP",val2="Grüne",country="Switzerland"){
  dat %>% 
    plyr::mutate(feature_order=as.numeric(factor(feature,ordered=T,levels=feature))) %>%
    ggplot(aes(x=feature,y=chi2))+
    geom_col(aes(color=party,fill=party),width=0.8)+
    scale_x_discrete(limits=rev(dat$feature),labels=rep("",20))+
    scale_color_manual(values=colors)+
    scale_fill_manual(values=colors)+
    geom_label(data=dat[dat$party==val1,],aes(y=0,x=feature,label=feature),hjust="right",label.size=0,alpha=0,size=2.6)+
    geom_label(data=dat[dat$party==val2,],aes(y=0,x=feature,label=feature),hjust="left",label.size=0,alpha=0,size=2.6)+
    geom_vline(xintercept=10.5,linetype="dotted")+
    geom_hline(yintercept=0,linetype="dotted")+
    scale_y_continuous()+
    theme_minimal()+coord_flip()+
    ggtitle(country)+ylab("")+xlab("")+guides(color="none",fill="none")
}

plot_contrast_ch<-function(dat=n_ch_public_sphere,val1="SVP",val2="Grüne",country="Switzerland"){
  dat %>% 
    plyr::mutate(feature_order=as.numeric(factor(feature,ordered=T,levels=feature))) %>%
    ggplot(aes(x=feature,y=chi2))+
    geom_col(aes(color=party,fill=party),width=0.8)+
    scale_x_discrete(limits=rev(dat$feature),labels=rep("",20))+
    scale_color_manual(values=c(colors[2],colors[1]))+
    scale_fill_manual(values=c(colors[2],colors[1]))+
    geom_label(data=dat[dat$party==val1,],aes(y=0,x=feature,label=feature),hjust="right",label.size=0,alpha=0,size=2.6)+
    geom_label(data=dat[dat$party==val2,],aes(y=0,x=feature,label=feature),hjust="left",label.size=0,alpha=0,size=2.6)+
    geom_vline(xintercept=10.5,linetype="dotted")+
    geom_hline(yintercept=0,linetype="dotted")+
    scale_y_continuous()+
    theme_minimal()+coord_flip()+
    ggtitle(country)+ylab("")+xlab("")+guides(color="none",fill="none")
}

at_transparency_plot <- plot_contrast(dat=n_at_transparency,country="Austria",val1="FPÖ",val2="Grüne")
de_transparency_plot <- plot_contrast(dat=n_de_transparency,country="Germany",val1="AfD",val2="Grüne")+geom_blank(aes(y=-chi2))
ch_transparency_plot <- plot_contrast_ch(dat=n_ch_transparency,country="Switzerland",val1="SVP",val2="Grüne")
at_participation_plot <- plot_contrast(dat=n_at_participation,country="Austria",val1="FPÖ",val2="Grüne")
de_participation_plot <- plot_contrast(dat=n_de_participation,country="Germany",val1="AfD",val2="Grüne")
ch_participation_plot <- plot_contrast_ch(dat=n_ch_participation,country="Switzerland",val1="SVP",val2="Grüne")
at_ind_liberties_plot <- plot_contrast(dat=n_at_ind_liberties,country="Austria",val1="FPÖ",val2="Grüne")
de_ind_liberties_plot <- plot_contrast(dat=n_de_ind_liberties,country="Germany",val1="AfD",val2="Grüne")+geom_blank(aes(y=-chi2))
ch_ind_liberties_plot <- plot_contrast_ch(dat=n_ch_ind_liberties,country="Switzerland",val1="SVP",val2="Grüne")
at_public_sphere_plot <- plot_contrast(dat=n_at_public_sphere,country="Austria",val1="FPÖ",val2="Grüne")
de_public_sphere_plot <- plot_contrast(dat=n_de_public_sphere,country="Germany",val1="AfD",val2="Grüne")
ch_public_sphere_plot <- plot_contrast_ch(dat=n_ch_public_sphere,country="Switzerland",val1="SVP",val2="Grüne")





plot_grid(at_transparency_plot,
          de_transparency_plot,
          ch_transparency_plot,
          #labels=c("Austria","Germany","Switzerland"),
          ncol=3)

 ggsave(filename="App_Figure4.pdf",
        path="temp",
        dpi=300,
        units="cm",
        width=20.32,
        height=12.7)


plot_grid(at_participation_plot,
          de_participation_plot,
          ch_participation_plot,
          #labels=c("Austria","Germany","Switzerland"),
          ncol=3)

 ggsave(filename="App_Figure6.pdf",
        path="temp",
        dpi=300,
        units="cm",
        width=20.32,
        height=12.7)



plot_grid(at_ind_liberties_plot,
          de_ind_liberties_plot,
          ch_ind_liberties_plot,
          #labels=c("Austria","Germany","Switzerland"),
          ncol=3)

 ggsave(filename="App_Figure5.pdf",
        path="temp",
        dpi=300,
        units="cm",
        width=20.32,
        height=12.7)



plot_grid(at_public_sphere_plot,
          de_public_sphere_plot,
          ch_public_sphere_plot,
          #labels=c("Austria","Germany","Switzerland"),
          ncol=3)

ggsave(filename="Figure5.pdf",
       path="temp",
       dpi=300,
       units="cm",
       width=20.32,
       height=12.7)



### Dictionary

principles<-names(dict_prinzip[[1]])[1:8]
keywords <- c( paste(tolower(dict_prinzip[[1]][[1]]),collapse=", "),
               paste(tolower(dict_prinzip[[1]][[2]]),collapse=", "),
               paste(tolower(dict_prinzip[[1]][[3]]),collapse=", "),
               paste(tolower(dict_prinzip[[1]][[4]]),collapse=", "),
               paste(tolower(dict_prinzip[[1]][[5]]),collapse=", "),
               paste(tolower(dict_prinzip[[1]][[6]]),collapse=", "),
               paste(tolower(dict_prinzip[[1]][[7]]),collapse=", "),
               paste(tolower(dict_prinzip[[1]][[8]]),collapse=", "))

data.frame(principles,keywords) %>% kable(caption="Democratic principle dictionary (German)",label="dict",booktabs=T,longtable=T,escape=T) %>% kable_styling() %>%
  column_spec(2, width = "10cm")


keywords.out <- data.frame(principles,keywords) 

write.csv(keywords.out, file="temp/App_Table1.csv")



bind_cols(list(Austria=n_at_transparency,
               Germany=n_de_transparency %>% rename_with(~paste0(.x,"1")),
               Switzerland=n_ch_transparency %>% rename_with(~paste0(.x,"2")))) %>%
  data.frame() %>%
  select(feature,chi2,feature1,chi21,feature2,chi22) %>%
  kable(caption="Most characteristic words for transparency",booktabs=T,digits=1,col.names=rep(c("Feature","Chi2"),3)) %>%
  kable_styling() %>% add_header_above(c("Austria"=2,"Germany"=2,"Switzerland"=2)) %>%
  group_rows("Populist right",1,10) %>% group_rows("Green alternative",11,20)

bind_cols(list(Austria=n_at_participation,
               Germany=n_de_participation %>% rename_with(~paste0(.x,"1")),
               Switzerland=n_ch_participation %>% rename_with(~paste0(.x,"2")))) %>%
  data.frame() %>%
  select(feature,chi2,feature1,chi21,feature2,chi22) %>%
  kable(caption="Most characteristic words for participation",booktabs=T,digits=1,col.names=rep(c("Feature","Chi2"),3)) %>%
  kable_styling() %>% add_header_above(c("Austria"=2,"Germany"=2,"Switzerland"=2)) %>%
  group_rows("Populist right",1,10) %>% group_rows("Green alternative",11,20)

bind_cols(list(Austria=n_at_ind_liberties,
               Germany=n_de_ind_liberties %>% rename_with(~paste0(.x,"1")),
               Switzerland=n_ch_ind_liberties %>% rename_with(~paste0(.x,"2")))) %>%
  data.frame() %>%
  select(feature,chi2,feature1,chi21,feature2,chi22) %>%
  kable(caption="Most characteristic words for individual liberties",booktabs=T,digits=1,col.names=rep(c("Feature","Chi2"),3)) %>%
  kable_styling() %>% add_header_above(c("Austria"=2,"Germany"=2,"Switzerland"=2)) %>%
  group_rows("Populist right",1,10) %>% group_rows("Green alternative",11,20)

bind_cols(list(Austria=n_at_public_sphere,
               Germany=n_de_public_sphere %>% rename_with(~paste0(.x,"1")),
               Switzerland=n_ch_public_sphere %>% rename_with(~paste0(.x,"2")))) %>%
  data.frame() %>%
  select(feature,chi2,feature1,chi21,feature2,chi22) %>%
  kable(caption="Most characteristic words for public sphere",booktabs=T,digits=1,col.names=rep(c("Feature","Chi2"),3)) %>%
  kable_styling() %>% add_header_above(c("Austria"=2,"Germany"=2,"Switzerland"=2)) %>%
  group_rows("Populist right",1,10) %>% group_rows("Green alternative",11,20)



## Accuracy

sample_coded <- read_excel("sample_coded.xlsx")

table(sample_coded$public_sphere)
table(sample_coded$ind_liberties)



smpl <- sample_coded %>%
  corpus() %>%
  tokens(remove_punct = T,
         remove_symbols = T,
         remove_separators = T,
         split_hyphens = T,
         remove_numbers = T,
         remove_url = T) %>%
  tokens_remove("\\d") %>% tokens_remove(c(stopwords("de"),"dass")) %>%
  tokens_tolower() %>%
  tokens_split(".") 

sample_toks_eval<-smpl %>% tokens_lookup(dict_prinzip,level=2)


sample_dist<-sample_toks_eval %>% dfm() %>% dfm_weight("boolean") %>% convert("data.frame") %>% bind_cols(sample_coded %>% rename_with(~paste0(.x,"1")))


res_public_sphere <- caret::confusionMatrix(factor(sample_dist$public_sphere),
                                            factor(sample_dist$public_sphere1),mode="everything",positive="1")
res_ind_liberties <- caret::confusionMatrix(factor(sample_dist$ind_liberties),
                                            factor(sample_dist$ind_liberties1),mode="everything",positive="1")
res_rule_of_law <- caret::confusionMatrix(factor(sample_dist$rule_of_law),
                                          factor(sample_dist$rule_of_law1),mode="everything",positive="1")
res_constraints <- caret::confusionMatrix(factor(sample_dist$constraints),
                                          factor(sample_dist$constraints1),mode="everything",positive="1")
res_transparency <- caret::confusionMatrix(factor(sample_dist$transparency),
                                           factor(sample_dist$transparency1),mode="everything",positive="1")
res_representation <- caret::confusionMatrix(factor(sample_dist$representation),
                                             factor(sample_dist$representation1),mode="everything",positive="1")
res_participation <- caret::confusionMatrix(factor(sample_dist$participation),
                                            factor(sample_dist$participation1),mode="everything",positive="1")
res_competition <- caret::confusionMatrix(factor(sample_dist$competition),
                                          factor(sample_dist$competition1),mode="everything",positive="1")




# Analysis Part with regression models (Lucas)

CHES <- read_dta("CHES_final_data.dta")

load("temp/at_agg1.RData")
count_df_hy$country <- "AT"
df_hy <- count_df_hy
count_df_q$country <- "AT"
df_q <- count_df_q

load("temp/ch_agg1.RData")
count_df_hy$country <- "CH"
df_hy <- rbind(df_hy,count_df_hy)
count_df_q$country <- "CH"
df_q <- rbind(df_q,count_df_q)


load("../Data/de_agg1.RData")
count_df_hy$country <- "GE"
df_hy <- rbind(df_hy,count_df_hy)
count_df_q$country <- "GE"
df_q <- rbind(df_q,count_df_q)

df_hy$partyfamily <- NA
df_hy$partyfamily[df_hy$party=="afd"] <- "Radical Right" 
df_hy$partyfamily[df_hy$party=="bdp"] <- "Conservatives" 
df_hy$partyfamily[df_hy$party=="cdu"] <- "Conservatives" 
df_hy$partyfamily[df_hy$party=="csu"] <- "Conservatives"
df_hy$partyfamily[df_hy$party=="cvp"] <- "Conservatives" 
df_hy$partyfamily[df_hy$party=="fdp"] <- "Liberals" 
df_hy$partyfamily[df_hy$party=="fdp_ch"] <- "Liberals" 
df_hy$partyfamily[df_hy$party=="fpo"] <- "Radical Right" 
df_hy$partyfamily[df_hy$party=="greenliberals"] <- "Liberals" 
df_hy$partyfamily[df_hy$party=="greens_ch"] <- "Left Libertarian" 
df_hy$partyfamily[df_hy$party=="gruene"] <- "Left Libertarian" 
df_hy$partyfamily[df_hy$party=="linke"] <- "Radical Left" 
df_hy$partyfamily[df_hy$party=="neos"] <- "Liberals" 
df_hy$partyfamily[df_hy$party=="ovp"] <- "Conservatives" 
df_hy$partyfamily[df_hy$party=="sp"] <- "Mainstream Left"
df_hy$partyfamily[df_hy$party=="spd"] <- "Mainstream Left"
df_hy$partyfamily[df_hy$party=="spo"] <- "Mainstream Left"
df_hy$partyfamily[df_hy$party=="svp"] <- "Radical Right"


df_hy$partyfamily <- factor(df_hy$partyfamily, levels = c("Radical Left", "Left Libertarian" , "Mainstream Left",
                                                          "Liberals","Conservatives" ,"Radical Right"))


df_hy$party[df_hy$party=="afd"] <- "AfD" 
df_hy$party[df_hy$party=="bdp"] <- "BDD" 
df_hy$party[df_hy$party=="cdu"] <- "CDU" 
df_hy$party[df_hy$party=="csu"] <- "CSU"
df_hy$party[df_hy$party=="cvp"] <- "CVP/PVC" 
df_hy$party[df_hy$party=="fdp"] <- "FDP" 
df_hy$party[df_hy$party=="fdp_ch"] <- "FDP/PLR" 
df_hy$party[df_hy$party=="fpo"] <- "FPO" 
df_hy$party[df_hy$party=="greenliberals"] <- "GLP/PVL" 
df_hy$party[df_hy$party=="greens_ch"] <- "GPS/PES" 
df_hy$party[df_hy$party=="gruene"] <- "GRUNE" 
df_hy$party[df_hy$party=="linke"] <- "Linke" 
df_hy$party[df_hy$party=="neos"] <- "NEOS" 
df_hy$party[df_hy$party=="ovp"] <- "OVP" 
df_hy$party[df_hy$party=="sp"] <- "SP/PS"
df_hy$party[df_hy$party=="spd"] <- "SPD"
df_hy$party[df_hy$party=="spo"] <- "SPO"
df_hy$party[df_hy$party=="svp"] <- "SVP/UDC"



CHES1 <- CHES[CHES$party=="AfD",][2,]
for (j in unique(CHES$party)){
  CHESa <- CHES[CHES$party==j,]
  max.y <- which.max(CHESa$year)
  CHESb <- CHESa[max.y,]
  CHES1 <- rbind(CHES1,CHESb)
}

CHES1 <- CHES1[-1,]
DATA1 <- merge(df_hy, CHES1, by="party")





# Table  in Appendix

DATA1$partyfamily <- relevel(DATA1$partyfamily, ref = "Conservatives")

m7 <- lm(100*public_sphere ~ factor(partyfamily) + factor(quarter) + factor(country.x), data=DATA1)
m8 <- lm(100*ind_liberties ~ factor(partyfamily) +  factor(quarter) + factor(country.x), data=DATA1)
m5 <- lm(100*rule_of_law ~ factor(partyfamily) + factor(quarter) + factor(country.x), data=DATA1)
m6 <- lm(100*constraints ~ factor(partyfamily) + factor(quarter) + factor(country.x), data=DATA1)
m4 <- lm(100*transparency ~ factor(partyfamily) +  factor(quarter) + factor(country.x), data=DATA1)
m3 <- lm(100*representation ~ factor(partyfamily) +   factor(quarter) + factor(country.x), data=DATA1)
m2 <- lm(100*participation ~ factor(partyfamily) +   factor(quarter) + factor(country.x), data=DATA1)
m1 <- lm(100*competition ~ factor(partyfamily) +  factor(quarter) + factor(country.x), data=DATA1)

principles <- c("Competition", "Participation", "Representation", "Transparency", "Rule of Law", 
                "Constraints", "Public Sphere","Ind. Liberties")

screenreg(list(m1,m2,m3,m4,m5,m6,m7,m8),
          omit.coef = c("quarter"),
          custom.model.names = principles)

texreg(list(m1,m2,m3,m4,m5,m6,m7,m8),
          omit.coef = c("quarter"),
          custom.model.names = principles, 
       file = "temp/App_Table2.txt")


# 
# wordreg(list(m1,m2,m3,m4,m5,m6,m7,m8),
#           omit.coef = c("quarter"),
#           custom.model.names = principles,
#         file = "temp/App_Table2.doc")

TAB1 <- matrix(NA,8,5)
TAB1[1,] <- coef(m1)[2:6]
TAB1[2,] <- coef(m2)[2:6]
TAB1[3,] <- coef(m3)[2:6]
TAB1[4,] <- coef(m4)[2:6]
TAB1[5,] <- coef(m5)[2:6]
TAB1[6,] <- coef(m6)[2:6]
TAB1[7,] <- coef(m7)[2:6]
TAB1[8,] <- coef(m8)[2:6]


TAB1.st <- matrix(NA,8,5)
TAB1.st[1,] <- summary(m1)$coefficients[2:6,2]
TAB1.st[2,] <- summary(m2)$coefficients[2:6,2]
TAB1.st[3,] <- summary(m3)$coefficients[2:6,2]
TAB1.st[4,] <- summary(m4)$coefficients[2:6,2]
TAB1.st[5,] <- summary(m5)$coefficients[2:6,2]
TAB1.st[6,] <- summary(m6)$coefficients[2:6,2]
TAB1.st[7,] <- summary(m7)$coefficients[2:6,2]
TAB1.st[8,] <- summary(m8)$coefficients[2:6,2]
# 



pdf("temp/Figure4.pdf",width=12,height=13) 
colors <- viridis(n=3, alpha = 1, begin = 0, end = 0.7, direction = -1, option = "D")
colors <- c(colors[2],colors[1],colors[3])
pa.label <- c("Radical Left","Left Libertarian","Social Democrats",  "Liberal", "Radical Right")
colorL <- c(colors[3],colors[2],colors[3],colors[3],colors[1])
par(mar=c(6.2, 6.1, 1.0, 6.1), mfrow=c(5,2))#,xpd=TRUE)#, mai=c(2,1,-1,0))

for (i in c(1,5,2,6,3,7,4,8,9,10)){
  
  if(i!=9&i!=10){
    plot(c(1:5),TAB1[i,], col=colors[3], pch=19, main=principles[i],
         bty="n", ylab="+/-", ylim=c(-3,9), xaxt='n', xlab="")#, xpd=TRUE)
    abline(h=0, lty=2, col=rgb(50,50,50,100,maxColorValue = 255))
    for (j in 1:5){
      segments(j,(TAB1[i,j]-2*TAB1.st[i,j]),j,TAB1[i,j]+2*TAB1.st[i,j], 
               col=colorL[j], lwd=2) 
    }
    
    points(2,TAB1[i,2], pch=17, cex=2, col=colors[2])
    points(5,TAB1[i,5], pch=15, cex=2, col=colors[1])
  }
  if(i==9|i==10){
    plot(c(1:5),rep(1,5), xaxt = 'n', yaxt = 'n', bty = 'n', pch = '', ylab = '', xlab = '', xpd=TRUE)
    
    for (k in 1:5){
      text(k,0.3,pa.label[k], font=2,
           srt = 90, cex=1.2, pos=4, xpd=TRUE)        
    }
    
  }
}
dev.off()





# table 5 in Appendix

DATA1.r <- DATA1[DATA1$party!="NEOS" & DATA1$party!="GLP/PVL", ]


m7 <- lm(100*public_sphere ~ factor(partyfamily) + factor(quarter) + factor(country.x), data=DATA1.r)
m8 <- lm(100*ind_liberties ~ factor(partyfamily) +  factor(quarter) + factor(country.x), data=DATA1.r)
m5 <- lm(100*rule_of_law ~ factor(partyfamily) + factor(quarter) + factor(country.x), data=DATA1.r)
m6 <- lm(100*constraints ~ factor(partyfamily) + factor(quarter) + factor(country.x), data=DATA1.r)
m4 <- lm(100*transparency ~ factor(partyfamily) +  factor(quarter) + factor(country.x), data=DATA1.r)
m3 <- lm(100*representation ~ factor(partyfamily) +   factor(quarter) + factor(country.x), data=DATA1.r)
m2 <- lm(100*participation ~ factor(partyfamily) +   factor(quarter) + factor(country.x), data=DATA1.r)
m1 <- lm(100*competition ~ factor(partyfamily) +  factor(quarter) + factor(country.x), data=DATA1.r)

screenreg(list(m1,m2,m3,m4,m5,m6,m7,m8),
          omit.coef = c("quarter"),
          custom.model.names = principles)


texreg(list(m1,m2,m3,m4,m5,m6,m7,m8),
       omit.coef = c("quarter"),
       custom.model.names = principles, 
       file = "temp/App_Table3.txt")



# Robustness of Dictionary Classification


sample_coded <- read_excel("sample_coded.xlsx")


smpl <- sample_coded %>%
  corpus() %>%
  tokens(remove_punct = T,
         remove_symbols = T,
         remove_separators = T,
         split_hyphens = T,
         remove_numbers = T,
         remove_url = T) %>%
  tokens_remove("\\d") %>% tokens_remove(c(stopwords("de"),"dass")) %>%
  tokens_tolower() %>%
  tokens_split(".") 

sample_toks_eval<-smpl %>% tokens_lookup(dict_prinzip,level=2)


sample_dist<-sample_toks_eval %>%
  dfm() %>%
  dfm_weight("boolean") %>% 
  convert("data.frame") %>% 
  bind_cols(sample_coded)



sample_dist %>% 
  plyr::mutate(ind_liberties_diff=paste(ind_liberties...3, "-",ind_liberties...17),
         participation_diff=paste(participation...8, "-",participation...22),
         transparency_diff=paste(transparency...6, "-",transparency...20),
         public_sphere_diff=paste(public_sphere...2, "-",public_sphere...16),
         rule_of_law_diff=paste(rule_of_law...4, "-",rule_of_law...18),
         constraints_diff=paste(constraints...5, "-",constraints...19),
         representation_diff=paste(representation...7, "-",representation...21),
         competition_diff=paste(competition...9, "-",competition...23)) %>%
  plyr::mutate(total=ind_liberties...3+participation...8+transparency...6+public_sphere...2+rule_of_law...4+
           constraints...5+representation...7+competition...9+
           ind_liberties...17+participation...22+transparency...20+public_sphere...16+rule_of_law...18+
           constraints...19+representation...21+competition...23) %>%
  filter(total!=0) %>% 
  filter(ind_liberties...3!=ind_liberties...17|
           participation...8!=participation...22|
           transparency...6!=transparency...20|
           public_sphere...2!=public_sphere...16|
           rule_of_law...4!=rule_of_law...18|
           constraints...5!=constraints...19|
           representation...7!=representation...21|
           competition...9!=competition...23) %>%
  select(title,party,ind_liberties_diff,participation_diff,transparency_diff,public_sphere_diff,rule_of_law_diff,constraints_diff,representation_diff,competition_diff,notes) %>% 
  datatable()





sample_dist$partyfamily <- NA
sample_dist$partyfamily[sample_dist$party=="afd"] <- "Radical Right" 
sample_dist$partyfamily[sample_dist$party=="bdp"] <- "Conservatives" 
sample_dist$partyfamily[sample_dist$party=="cdu"] <- "Conservatives" 
sample_dist$partyfamily[sample_dist$party=="csu"] <- "Conservatives"
sample_dist$partyfamily[sample_dist$party=="cvp"] <- "Conservatives" 
sample_dist$partyfamily[sample_dist$party=="fdp"] <- "Liberals" 
sample_dist$partyfamily[sample_dist$party=="fdp_ch"] <- "Liberals" 
sample_dist$partyfamily[sample_dist$party=="fpo"] <- "Radical Right" 
sample_dist$partyfamily[sample_dist$party=="greenliberals"] <- "Liberals" 
sample_dist$partyfamily[sample_dist$party=="greens_ch"] <- "Left Libertarian" 
sample_dist$partyfamily[sample_dist$party=="gruene"] <- "Left Libertarian" 
sample_dist$partyfamily[sample_dist$party=="linke"] <- "Radical Left" 
sample_dist$partyfamily[sample_dist$party=="neos"] <- "Liberals" 
sample_dist$partyfamily[sample_dist$party=="ovp"] <- "Conservatives" 
sample_dist$partyfamily[sample_dist$party=="sp"] <- "Mainstream Left"
sample_dist$partyfamily[sample_dist$party=="spd"] <- "Mainstream Left"
sample_dist$partyfamily[sample_dist$party=="spo"] <- "Mainstream Left"
sample_dist$partyfamily[sample_dist$party=="svp"] <- "Radical Right"




# Individual Liberties
N <- length(sample_dist$ind_liberties...3)
collector.ind <- rep(NA,N)
for (i in 1:N){
  picker <- sample(1:N,N,replace = TRUE)
  measure <- sample_dist$ind_liberties...3[picker]
  measure1 <- sample_dist$ind_liberties...17[picker]
  collector.ind[i] <- sum(diag(table(measure,measure1)))/N
}


# Participation
N <- length(sample_dist$ind_liberties...3)
collector.par <- rep(NA,N)
for (i in 1:N){
  picker <- sample(1:N,N,replace = TRUE)
  measure <- sample_dist$participation...8[picker]
  measure1 <- sample_dist$participation...22[picker]
  collector.par[i] <- sum(diag(table(measure,measure1)))/N
}

# transparency
N <- length(sample_dist$ind_liberties...3)
collector.tra <- rep(NA,N)
for (i in 1:N){
  picker <- sample(1:N,N,replace = TRUE)
  measure <- sample_dist$transparency...6[picker]
  measure1 <- sample_dist$transparency...20[picker]
  collector.tra[i] <- sum(diag(table(measure,measure1)))/N
}

# public sphere
N <- length(sample_dist$ind_liberties...3)
collector.sph <- rep(NA,N)
for (i in 1:N){
  picker <- sample(1:N,N,replace = TRUE)
  measure <- sample_dist$public_sphere...2[picker]
  measure1 <- sample_dist$public_sphere...16[picker]
  collector.sph[i] <- sum(diag(table(measure,measure1)))/N
}


# rule of law
N <- length(sample_dist$ind_liberties...3)
collector.rol <- rep(NA,N)
for (i in 1:N){
  picker <- sample(1:N,N,replace = TRUE)
  measure <- sample_dist$rule_of_law...4[picker]
  measure1 <- sample_dist$rule_of_law...18[picker]
  collector.rol[i] <- sum(diag(table(measure,measure1)))/N
}



# constraints
N <- length(sample_dist$constraints...5)
collector.con <- rep(NA,N)
for (i in 1:N){
  picker <- sample(1:N,N,replace = TRUE)
  measure <- sample_dist$constraints...5[picker]
  measure1 <- sample_dist$constraints...19[picker]
  collector.con[i] <- sum(diag(table(measure,measure1)))/N
}

# representation
N <- length(sample_dist$constraints...5)
collector.rep <- rep(NA,N)
for (i in 1:N){
  picker <- sample(1:N,N,replace = TRUE)
  measure <- sample_dist$representation...7[picker]
  measure1 <- sample_dist$representation...21[picker]
  collector.rep[i] <- sum(diag(table(measure,measure1)))/N
}

# competition
N <- length(sample_dist$constraints...5)
collector.com <- rep(NA,N)
for (i in 1:N){
  picker <- sample(1:N,N,replace = TRUE)
  measure <- sample_dist$competition...9[picker]
  measure1 <- sample_dist$competition...23[picker]
  collector.com[i] <- sum(diag(table(measure,measure1)))/N
}


Qcatcher <- matrix(NA,3,8)
Qcatcher[,1] <- quantile(collector.ind, c(0.025,.5,.975))
Qcatcher[,2] <- quantile(collector.par, c(0.025,.5,.975))
Qcatcher[,3] <- quantile(collector.tra, c(0.025,.5,.975))
Qcatcher[,4] <- quantile(collector.sph, c(0.025,.5,.975))
Qcatcher[,5] <- quantile(collector.rol, c(0.025,.5,.975))
Qcatcher[,6] <- quantile(collector.con, c(0.025,.5,.975))
Qcatcher[,7] <- quantile(collector.rep, c(0.025,.5,.975))
Qcatcher[,8] <- quantile(collector.com, c(0.025,.5,.975))

dimen <- c("Individual Liberties",
           "Participation",
           "Transparency",
           "Public Sphere",
           "Rule of Law",
           "Constraints",
           "Representation",
           "Competition")

plot(0,0,xlim=c(.5,8.5),ylim=c(0.78,1), bty="n", ylab="Accuracy", 
     xlab="Principles", xaxt="n")
for (i in 1:8){
  segments(i,Qcatcher[1,i],i,Qcatcher[3,i])
  points(i,Qcatcher[2,i],pch=19)
  text(i,.83,dimen[i],srt=90, cex=.8,pos = 1)
}





### By parties - within one principle


# Individual Liberties
N <- length(sample_dist$ind_liberties...3)
collector.ind <- matrix(NA,N,length(table(sample_dist$partyfamily)))

pfm <- names(table(sample_dist$partyfamily))

counti <- 0
for (f in pfm){
  counti <- counti +1
  df <- sample_dist[sample_dist$partyfamily==f,]
  for (i in 1:dim(df)[1]){
    picker <- sample(1:dim(df)[1],dim(df)[1],replace = TRUE)
    measure <- df$ind_liberties...3[picker]
    measure1 <- df$ind_liberties...17[picker]
    collector.ind[i,counti] <- sum(diag(table(measure,measure1)))/dim(df)[1]
  }
}

out.block <- apply(collector.ind, 2, quantile,c(0.025,.5,0.975), na.rm=TRUE)





## Plots Robustness




sample_dist <- sample_dist %>% 
 rename("ind_liberties" = "ind_liberties...3",  "ind_liberties1" = "ind_liberties...17",
        "participation" = "participation...8", "participation1" = "participation...22",
        "transparency" = "transparency...6", "transparency1" = "transparency...20",
        "public_sphere" = "public_sphere...2", "public_sphere1" = "public_sphere...16",
        "rule_of_law" = "rule_of_law...4", "rule_of_law1" = "rule_of_law...18",
        "constraints" = "constraints...5", "constraints1" = "constraints...19",
        "representation" = "representation...7", "representation1"= "representation...21",
        "competition" = "competition...9", "competition1" ="competition...23" )



pfm <- names(table(sample_dist$partyfamily))

princ <- c("ind_liberties", "participation",
           "transparency","public_sphere","rule_of_law","constraints",
           "representation","competition")
for (p in princ){
  
  # Principle
  
  NN <- length(sample_dist$ind_liberties)
  collector.overall <- rep(NA,N)
  for (i in 1:NN){
    picker <- sample(1:NN,NN,replace = TRUE)
    measure <- eval(parse(text=eval(paste0("sample_dist$",p,"[picker]"))))
    measure1 <- eval(parse(text=eval(paste0("sample_dist$",p,"1[picker]"))))
    collector.overall[i] <- sum(diag(table(measure,measure1)))/N
  }
  
  
  N <- length(sample_dist$ind_liberties)
  collector.ind <- matrix(NA,N,length(table(sample_dist$partyfamily)))
  
  
  counti <- 0
  for (f in pfm){
    counti <- counti +1
    df <- sample_dist[sample_dist$partyfamily==f,]
    for (i in 1:dim(df)[1]){
      picker <- sample(1:dim(df)[1],dim(df)[1],replace = TRUE)
      measure <- eval(parse(text=eval(paste0("df$",p,"[picker]"))))
      measure1 <- eval(parse(text=eval(paste0("df$",p,"1[picker]"))))
      collector.ind[i,counti] <- sum(diag(table(measure,measure1)))/dim(df)[1]
    }
  }
  
  out.block <- apply(collector.ind, 2, quantile,c(0.025,.5,0.975), na.rm=TRUE)
  
  png(paste0("temp/App_Figure_acc_",p,".jpg"))
  plot(0,0,xlim=c(.5,6.5),ylim=c(0.5,1), bty="n", ylab=p, 
       xlab="Party Families", xaxt="n")
  for (i in 1:6){
    segments(i,out.block[1,i],i,out.block[3,i])
    points(i,out.block[2,i],pch=19)
    text(i,.59,pfm[i],srt=90, cex=.8)
  }
  QQQ <- quantile(collector.overall,c(0.025,0.5,0.975))
  rect(0,QQQ[1],7,QQQ[3], col=rgb(0,0,255,50,maxColorValue = 255),border=NA)
  abline(h=QQQ[1],lty=2,col="blue")
  abline(h=QQQ[3],lty=2,col="blue")
  abline(h=QQQ[2],lty=1,col="blue")
  dev.off()
  
}





